from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from pyquery import PyQuery as pq
import datetime
import json
import re
import time
import os
import csv
import pymysql
import sys

# 数据库配置
host = '127.0.0.1' # localhost:本地数据库
user = 'root' # 用户名
password = '123456' # 密码
database = 'crawler' # 数据库名称
charset = 'utf8' # 编码方式

# 创建数据库连接
conn = pymysql.connect(host=host, user=user, password=password
                           , database=database, charset=charset)
cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)


# 爬虫设置
browser=webdriver.Chrome()
browser.maximize_window()   #将窗口最大化
wait=WebDriverWait(browser, 10)

# 搜索
def search(id,url):
    browser.get(url)
    jiexi_page(id)

def jiexi_page(id):
    wait.until(
        EC.presence_of_element_located((By.CSS_SELECTOR,".summary-price-wrap"))    #判断是否加载成功
    )
    now_time = datetime.datetime.now()
    html=browser.page_source
    doc=pq(html)
    price=doc(".p-price").find(".price").text()   #遍历
    sql = "insert into crawler_monitoringgoodspricehistory (price,monitoring_goods_id,add_time) values (%s,%s,%s)"
    cursor.execute(sql,tuple([price,id,now_time]))
    conn.commit()

def main():
    sql = "select crawler_id,address from crawler_monitoringgoods"
    cursor.execute(sql)
    results = cursor.fetchall()
    for row in results:
        search(row['crawler_id'],row['address'])
        time.sleep(1)

if __name__=="__main__":
    main()